"""
  Copyright (c) 2024 Northeastern University (China)
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""
# 任务：对于目标代码划分的每个模块建立集合（.h文件相对路径集合，全局变量集合）
# 输入：由Partition_module划分好的模块字典module_dic
# 输出：target_features={
#      'name':'oss1',
#      'feature':[.h路径，全局变量和函数的定义代码]

import json
import math
import multiprocessing
import os
import re
import subprocess
import threading
import traceback
from concurrent.futures import ThreadPoolExecutor

from tqdm import tqdm

from src.common_utils import configure
from src.common_utils.utils_file import read_jl, read_file_with_encoding_lines

ctagsPath= configure.CONFIG['ctags_path']

class ExtractionFeature:
    def removeComment(self,string):
        # Code for removing C/C++ style comments. (Imported from VUDDY and ReDeBug.)
        # ref: https://github.com/squizz617/vuddy
        c_regex = re.compile(
            r'(?P<comment>//.*?$|[{}]+)|(?P<multilinecomment>/\*.*?\*/)|(?P<noncomment>\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)',
            re.DOTALL | re.MULTILINE)
        return ''.join([c.group('noncomment') for c in c_regex.finditer(string) if c.group('noncomment')])
    def headfiels_feature(self,module_dic,target_features):
        """
        提取每个模块.h文件相对路径
        :return:
        """
        for moudle in module_dic:
            paths=[]
            paths=moudle['path']
            # for path in moudle['path']:
            #     if path.endswith(".h"):
            #         if path not in paths:
            #             paths.append(path)
            oss_feature={
                'name':moudle['name'],
                'head_file_feature':paths,
                'global_feature':[],
                'indirect_feature':[],#间接特征
                'global_var':[],
                'global_func':[]
            }
            target_features.append(oss_feature)
        return target_features

    def ctags_other_feature_main(self,module_dic, target_features, dependency_dic,target_path,numProcess):
        cnt = 0
        manager = multiprocessing.Manager()
        for moudle in module_dic:
            paths=moudle['path']
            processes = []
            lock=manager.Lock()
            global_feature= manager.list(target_features[cnt]['global_feature'])
            global_var=manager.list(target_features[cnt]['global_var'])
            global_func=manager.list(target_features[cnt]['global_func'])
            indirect_feature=manager.list(target_features[cnt]['indirect_feature'])
            orig_extern_dependency = manager.list(dependency_dic[cnt]['orig_extern_dependency'])
            orig_extern_dependency_statement = manager.list(dependency_dic[cnt]['orig_extern_dependency_statement'])
            with multiprocessing.Pool(processes=numProcess) as pool:
                pool.map(
                    self.ctags_other_feature_detail,
                    [
                        (
                        idx + 1, list_arr, target_path, global_feature, global_var,global_func,indirect_feature,orig_extern_dependency,orig_extern_dependency_statement, lock)
                        for idx, list_arr in enumerate(self.split_list(paths, numProcess))
                    ]
                )
            # for idx, list_arr in enumerate(self.split_list(paths, numProcess)):  # 分割任务列表并获取进程编号
            #     process = multiprocessing.Process(target=self.ctags_other_feature_detail,
            #                                       args=(idx + 1, list_arr,target_path, cnt,target_features_manager, dependency_dic_manager,lock))
            #     processes.append(process)
            # for process in processes:
            #     process.start()
            # for process in processes:
            #     process.join()  # 等待所有子进程结束后再执行主进程的代码
            target_features[cnt]['global_feature'] = list(global_feature)
            target_features[cnt]['global_var'] = list(global_var)
            target_features[cnt]['global_func'] = list(global_func)
            target_features[cnt]['indirect_feature'] =list(sorted(
               list(indirect_feature), key=lambda x: x['weight'],
                reverse=True))
            dependency_dic[cnt]['orig_extern_dependency'] = list(orig_extern_dependency)
            dependency_dic[cnt]['orig_extern_dependency_statement'] = list(orig_extern_dependency_statement)
            cnt+=1
        return target_features,dependency_dic
    def ctags_other_feature_detail(self,args):
        idx, listarr, target_path, global_feature, global_var,global_func,indirect_feature,orig_extern_dependency,orig_extern_dependency_statement, lock=args
        for path in tqdm(listarr,total=len(listarr),desc=f'{idx}ctags信息提取进度：'):
            filePath = os.path.join(target_path, path.strip("\\"))
            try:
                # Execute Ctgas command
                TagsList = subprocess.check_output(ctagsPath + ' -f - --kinds-C=* --fields=+KS "' + filePath + '"',
                                                   stderr=subprocess.STDOUT, shell=True).decode()
                allGlobals = str(TagsList).split('\n')
                for i in allGlobals:
                    if not i.startswith("!_TAG"):
                        feature = None  # 全局特征
                        dependency = None  # 依赖信息
                        statement = None  # 依赖语句
                        Identifier = None  # 标识符
                        elemList = re.sub(r'[\t\s ]{2,}', '', i)
                        elemList = elemList.split('\t')
                        if i != '' and len(elemList) >= 4:
                            if re.compile(r'(variable)').fullmatch(elemList[3]):  # 标识符是变量定义
                                feature_reg = elemList[2]
                                pattern = r"\/\^\s*(friend|static)"
                                if re.match(pattern, feature_reg):
                                    pass
                                else:
                                    if elemList[4].startswith("typeref:typename:") and not feature_reg.startswith(
                                            '/^static'):  # 这是为了防止静态变量出现
                                        # print(elemList)
                                        feature = {
                                            'feature_type': 'variable',
                                            'name': elemList[0],
                                            'type': elemList[-1].replace("typeref:typename:", "").strip('\r'),
                                            'filepath': str(path)
                                        }

                            elif re.compile(r'(function)').fullmatch(elemList[3]):  # 标识符是函数定义
                                feature_reg = elemList[2]
                                pattern = r"\/\^\s*(friend|static)"
                                if re.match(pattern, feature_reg):
                                    pass
                                else:
                                    if elemList[4].startswith("typeref:typename:") and not feature_reg.startswith(
                                            '/^static'):  # 说明这个函数不是类得成员
                                        feature = {
                                            'feature_type': 'function',
                                            'name': elemList[0],
                                            'type': elemList[4].replace("typeref:typename:", ""),
                                            'signature': elemList[-1].replace("signature:", "").strip('\r'),
                                            'filepath': str(path)
                                        }

                            elif re.compile(r'(externvar)').fullmatch(elemList[3]):  # 标识符是外部变量extern
                                dependency = {
                                    'statement_type': 'variable',
                                    'name': elemList[0],
                                    'type': elemList[-1].replace("typeref:typename:", "").strip('\r'),
                                    'filepath': str(path)
                                }
                                statement = elemList[2].strip("{$/;")
                                statement = statement.strip("/^")
                            elif re.compile(r'(prototype)').fullmatch(elemList[3]):  # 标识符是函数的原型，有可能存在extern
                                if elemList[4].startswith("typeref:typename:") and "extern " in elemList[
                                    2]:  # 说明这个函数原型不是在类或者结构体中
                                    dependency = {
                                        'statement_type': 'function',
                                        'name': elemList[0],
                                        'type': elemList[4].replace("typeref:typename:", ""),
                                        'signature': elemList[-1].replace("signature:", "").strip('\r'),
                                        'filepath': str(path)
                                    }
                                    statement = elemList[2]
                            # 以下提取标识符
                            # 只提取类，结构体，枚举
                            elif re.compile(r'(class)').fullmatch(elemList[3]):  # 标识符是函数定义
                                feature_reg = elemList[2]
                                inner_id = False
                                namespace = ""
                                if not elemList[0].islower():
                                    for i in range(3, len(elemList)):
                                        if elemList[i].strip().startswith("namespace"):
                                            namespace = elemList[i].replace("namespace:",
                                                                            "").strip()  # 类似于：namespace:first_space::second_space
                                            break
                                        elif elemList[i].strip().startswith("struct:") or elemList[
                                            i].strip().startswith(
                                            "class:") or elemList[i].strip().startswith("enum:") or elemList[
                                            i].strip().startswith("function:"):
                                            inner_id = True
                                            break
                                    if not inner_id:
                                        Identifier = {
                                            'feature_type': 'class',
                                            'name': elemList[0],
                                            'filepath': path,
                                            'namespace': namespace,
                                            'weight': 2
                                        }
                            elif re.compile(r'(struct)').fullmatch(elemList[3]):  # 标识符是函数定义
                                feature_reg = elemList[2]
                                inner_id = False
                                namespace = ""

                                if not elemList[0].islower():
                                    for i in range(3, len(elemList)):
                                        if elemList[i].strip().startswith("namespace"):
                                            namespace = elemList[i].replace("namespace:",
                                                                            "").strip()  # 类似于：namespace:first_space::second_space
                                            break
                                        elif elemList[i].strip().startswith("struct:") or elemList[
                                            i].strip().startswith("class:") or elemList[i].strip().startswith(
                                            "enum:") or elemList[i].strip().startswith("function:"):
                                            inner_id = True
                                            break
                                    if not inner_id:
                                        Identifier = {
                                            'feature_type': 'struct',
                                            'name': elemList[0],
                                            'filepath': path,
                                            'namespace': namespace,
                                            'weight': 2
                                        }
                            elif re.compile(r'(enum)').fullmatch(elemList[3]):  # 标识符是变量定义
                                feature_reg = elemList[2]
                                inner_id = False
                                namespace = ""
                                if not elemList[0].islower():
                                    print(elemList)
                                    for i in range(3, len(elemList)):
                                        if elemList[i].strip().startswith("namespace"):
                                            namespace = elemList[i].replace("namespace:",
                                                                            "").strip()  # 类似于：namespace:first_space::second_space
                                            break
                                        elif elemList[i].strip().startswith("struct:") or elemList[
                                            i].strip().startswith(
                                            "class:") or elemList[i].strip().startswith("enum:") or elemList[
                                            i].strip().startswith("function:"):
                                            inner_id = True
                                            break
                                    if not inner_id:
                                        Identifier = {
                                            'feature_type': 'enum',
                                            'name': elemList[0],
                                            'filepath': path,
                                            'namespace': namespace,
                                            'weight': 2
                                        }
                            with lock:
                                if feature is not None:
                                    if feature in global_feature:
                                        pass
                                    else:
                                        if feature['feature_type'] == 'variable':
                                            global_var.append(elemList[0])
                                        elif feature['feature_type'] == 'function':
                                            global_func.append(elemList[0])
                                        global_feature.append(feature)
                                if Identifier is not None:
                                    indirect_feature.append(Identifier)
                                if dependency is not None and dependency not in orig_extern_dependency_statement:
                                    orig_extern_dependency.append(dependency)
                                    orig_extern_dependency_statement.append(statement)
            except subprocess.CalledProcessError as e:
                print("Parser Error:", e)
                traceback.print_exc()
                continue
            except Exception as e:
                print("Subprocess failed", e)
                traceback.print_exc()
                continue

    def ctags_other_feature(self,module_dic, target_features, dependency_dic,target_path):
        """
        执行ctags命令，根据生成的索引信息，找到需要的信息
        1.找到全局变量的定义，确定每个模块都定义哪些全局变量
        2.a.找到每个模块中的externvar，明确是使用的本源文件外部的变量
          b.找到prototype，这是说明在本源文件中使用了一个函数的原型，函数调用也是这个类型，表明有可能是使用的本源文件之外的函数
        3.找到结构体，类，枚举类型的定义，同时记录命名空间和1.所找的全局变量有不同，这是为了知道每个模块中都定义了哪些这三类对象，有可能在其他模块中会使用到，这是为了识别间接依赖所建立的特征和依赖信息
        @param module_dic:
        @param target_features:
        @param dependency_dic:
        @return:
        """
        cnt = 0
        for moudle in module_dic:
            for path in moudle['path']:
                filePath = os.path.join(target_path, path.strip("\\"))
                try:
                    # Execute Ctgas command
                    TagsList = subprocess.check_output(ctagsPath + ' -f - --kinds-C=* --fields=+KS "' + filePath + '"',
                                                       stderr=subprocess.STDOUT, shell=True).decode()
                    allGlobals = str(TagsList).split('\n')
                    for i in allGlobals:
                        if not i.startswith("!_TAG"):
                            feature = None  # 全局特征
                            dependency = None  # 依赖信息
                            statement = None  # 依赖语句
                            Identifier = None  # 标识符
                            elemList = re.sub(r'[\t\s ]{2,}', '', i)
                            elemList = elemList.split('\t')
                            if i != '' and len(elemList) >= 4:
                                if re.compile(r'(variable)').fullmatch(elemList[3]):  # 标识符是变量定义
                                    feature_reg = elemList[2]
                                    pattern = r"\/\^\s*(friend|static)"
                                    if re.match(pattern, feature_reg):
                                        pass
                                    else:
                                        if elemList[4].startswith("typeref:typename:") and not feature_reg.startswith(
                                                '/^static'):  # 这是为了防止静态变量出现
                                            # print(elemList)
                                            feature = {
                                                'feature_type': 'variable',
                                                'name': elemList[0],
                                                'type': elemList[-1].replace("typeref:typename:", "").strip('\r'),
                                                'filepath': str(path)
                                            }

                                elif re.compile(r'(function)').fullmatch(elemList[3]):  # 标识符是函数定义
                                    feature_reg = elemList[2]
                                    pattern = r"\/\^\s*(friend|static)"
                                    if re.match(pattern, feature_reg):
                                        pass
                                    else:
                                        if elemList[4].startswith("typeref:typename:") and not feature_reg.startswith(
                                                '/^static'):  # 说明这个函数不是类得成员
                                            feature = {
                                                'feature_type': 'function',
                                                'name': elemList[0],
                                                'type': elemList[4].replace("typeref:typename:", ""),
                                                'signature': elemList[-1].replace("signature:", "").strip('\r'),
                                                'filepath': str(path)
                                            }

                                elif re.compile(r'(externvar)').fullmatch(elemList[3]):  # 标识符是外部变量extern
                                    dependency = {
                                        'statement_type': 'variable',
                                        'name': elemList[0],
                                        'type': elemList[-1].replace("typeref:typename:", "").strip('\r'),
                                        'filepath': str(path)
                                    }
                                    statement = elemList[2].strip("{$/;")
                                    statement = statement.strip("/^")
                                elif re.compile(r'(prototype)').fullmatch(elemList[3]):  # 标识符是函数的原型，有可能存在extern
                                    if elemList[4].startswith("typeref:typename:") and "extern " in elemList[
                                        2]:  # 说明这个函数原型不是在类或者结构体中
                                        dependency = {
                                            'statement_type': 'function',
                                            'name': elemList[0],
                                            'type': elemList[4].replace("typeref:typename:", ""),
                                            'signature': elemList[-1].replace("signature:", "").strip('\r'),
                                            'filepath': str(path)
                                        }
                                        statement = elemList[2]
                                # 以下提取标识符
                                # 只提取类，结构体，枚举
                                elif re.compile(r'(class)').fullmatch(elemList[3]):  # 标识符是函数定义
                                    feature_reg = elemList[2]
                                    inner_id = False
                                    namespace = ""
                                    if not elemList[0].islower():
                                        for i in range(3, len(elemList)):
                                            if elemList[i].strip().startswith("namespace"):
                                                namespace = elemList[i].replace("namespace:",
                                                                                "").strip()  # 类似于：namespace:first_space::second_space
                                                break
                                            elif elemList[i].strip().startswith("struct:") or elemList[
                                                i].strip().startswith(
                                                    "class:") or elemList[i].strip().startswith("enum:") or elemList[i].strip().startswith("function:"):
                                                inner_id = True
                                                break
                                        if not inner_id:
                                            Identifier = {
                                                'feature_type': 'class',
                                                'name': elemList[0],
                                                'filepath': path,
                                                'namespace': namespace,
                                                'weight': 2
                                            }
                                elif re.compile(r'(struct)').fullmatch(elemList[3]):  # 标识符是函数定义
                                    feature_reg = elemList[2]
                                    inner_id = False
                                    namespace = ""

                                    if not elemList[0].islower():
                                        for i in range(3, len(elemList)):
                                            if elemList[i].strip().startswith("namespace"):
                                                namespace = elemList[i].replace("namespace:",
                                                                                "").strip()  # 类似于：namespace:first_space::second_space
                                                break
                                            elif elemList[i].strip().startswith("struct:") or elemList[
                                                i].strip().startswith("class:") or elemList[i].strip().startswith("enum:")or elemList[i].strip().startswith("function:"):
                                                inner_id = True
                                                break
                                        if not inner_id:
                                            Identifier = {
                                                'feature_type': 'struct',
                                                'name': elemList[0],
                                                'filepath': path,
                                                'namespace': namespace,
                                                'weight': 2
                                            }
                                elif re.compile(r'(enum)').fullmatch(elemList[3]):  # 标识符是变量定义
                                    feature_reg = elemList[2]
                                    inner_id = False
                                    namespace = ""
                                    if not elemList[0].islower():
                                        print(elemList)
                                        for i in range(3, len(elemList)):
                                            if elemList[i].strip().startswith("namespace"):
                                                namespace = elemList[i].replace("namespace:",
                                                                                "").strip()  # 类似于：namespace:first_space::second_space
                                                break
                                            elif elemList[i].strip().startswith("struct:") or elemList[
                                                i].strip().startswith(
                                                    "class:") or elemList[i].strip().startswith("enum:")or elemList[i].strip().startswith("function:"):
                                                inner_id = True
                                                break
                                        if not inner_id:
                                            Identifier = {
                                                'feature_type': 'enum',
                                                'name': elemList[0],
                                                'filepath': path,
                                                'namespace': namespace,
                                                'weight': 2
                                            }
                                if feature is not None:
                                    if feature in target_features[cnt]['global_feature']:
                                        pass
                                    else:
                                        if feature['feature_type'] == 'variable':
                                            target_features[cnt]['global_var'].append(elemList[0])
                                        elif feature['feature_type'] == 'function':
                                            target_features[cnt]['global_func'].append(elemList[0])
                                        target_features[cnt]['global_feature'].append(feature)
                                if Identifier is not None:
                                    target_features[cnt]['indirect_feature'].append(Identifier)
                                    target_features[cnt]['indirect_feature'] = sorted(
                                        target_features[cnt]['indirect_feature'], key=lambda x: x['weight'],
                                        reverse=True)
                                if dependency is not None and dependency not in dependency_dic[cnt][
                                    'orig_extern_dependency']:
                                    dependency_dic[cnt]['orig_extern_dependency'].append(dependency)
                                    dependency_dic[cnt]['orig_extern_dependency_statement'].append(statement)
                except subprocess.CalledProcessError as e:
                    print("Parser Error:", e)
                    traceback.print_exc()
                    continue
                except Exception as e:
                    print("Subprocess failed", e)
                    traceback.print_exc()
                    continue
            cnt = cnt + 1
        return target_features, dependency_dic
    def split_list(self, list, n):
        """
        将数据n等分，分发给n个线程执行

        :param list: 原始完整list
        :param n: 切分数量，即线程数量
        :return: 切分后的list起始位置
        """
        step_count = math.ceil(len(list) / n)
        for i in range(0, len(list), step_count):
            end = i + step_count
            if end > len(list):
                end = len(list)
            # print(f"yield: {i}:{end}")
            yield list[i:end]
    def scan_include_dependency_main(self,moudle_dic,numProcess, dependency_dic,target_path):
        """
        多进程扫描include语句的主函数
        """
        manager = multiprocessing.Manager()
        for moudle in moudle_dic:
            oss_name = moudle['name']
            paths = moudle['path']
            include_dependency = manager.list()
            extern_dependency = []
            include_dependency_statement = manager.list()
            extern_dependency_statement = []
            processes = []
            lock=manager.Lock()
            # 创建进程池
            with multiprocessing.Pool(processes=numProcess) as pool:
                # 将任务分配给进程池
                pool.map(self.scan_include_dependency_detail,
                         [(idx + 1, list_arr, lock, target_path, include_dependency_statement, include_dependency)for
                          idx, list_arr in enumerate(self.split_list(paths, numProcess))])
            # for idx, list_arr in enumerate(self.split_list(paths, numProcess)):  # 分割任务列表并获取进程编号
            #     process = multiprocessing.Process(target=self.scan_include_dependency_detail,
            #                                       args=(idx + 1, list_arr, lock,target_path,include_dependency_statement,include_dependency))
            #     processes.append(process)
            # for process in processes:
            #     process.start()
            # for process in processes:
            #     process.join()  # 等待所有子进程结束后再执行主进程的代码
            include_dependency=list(include_dependency)
            include_dependency_statement=list(include_dependency_statement)
            oss_dependency = {
                'name': oss_name,
                'orig_include_dependency': include_dependency,
                'orig_extern_dependency': extern_dependency,
                'orig_include_dependency_statement': include_dependency_statement,
                'orig_extern_dependency_statement': extern_dependency_statement,
                'include_dependency': include_dependency.copy(),
                'extern_dependency': extern_dependency.copy(),
            }
            dependency_dic.append(oss_dependency)
        return dependency_dic
    def scan_include_dependency_detail(self, args):
        """
        扫描Include的子进程单线任务
        """
        idx, listarr, lock, target_path, include_dependency_statement, include_dependency=args
        for path in tqdm(listarr,total=len(listarr),desc=f'{idx}include语句扫描进度：'):
            filepath = os.path.join(target_path, path.strip("\\"))
            try:
                lines = read_file_with_encoding_lines(filepath)
                for orig_l in lines:
                    try:
                        l = orig_l
                        l = self.removeComment(l)
                        l = l.strip()
                        if l == "":
                            continue
                        if l.startswith("#include "):
                            include_statements = re.findall(r'^#include\s*[<"][^>"]*[>"]', l)
                            for include in include_statements:
                                include = include.replace("/", "\\")
                                include = re.sub(r"(#include)\s+", "", include)
                                include = re.sub(r'^["<]|[">]$', '', include)
                                include = include.strip(";")
                                include = include.strip()  # 这时应该只剩下引入的头文件相对路径了,不需要补全成绝对路径，因为会和编译和构建系系统的配之有关
                                with lock:
                                    if include not in include_dependency_statement:  # include引入的可以不是.h后缀的文件
                                        include_dependency_statement.append(orig_l)
                                        dependency_info = {
                                            'statement': include,
                                            'filepath': path
                                        }
                                        if dependency_info not in include_dependency:
                                            include_dependency.append(dependency_info)

                    except Exception as e:
                        print(f"error: {str(e)}")
                        traceback.print_exc()
                        continue
            except Exception as e:
                print(f"error: {str(e)}")
                traceback.print_exc()
                continue


    def scan_include_dependency(self, moudle_dic, dependency_dic,target_path):
        """
        扫描每个模块中的include语句
        @param moudle_dic:
        @param dependency_dic:
        @return:
        """
        for moudle in moudle_dic:
            oss_name = moudle['name']
            paths = moudle['path']
            include_dependency = []
            extern_dependency = []
            include_dependency_statement = []
            extern_dependency_statement = []
            for path in paths:
                filepath = os.path.join(target_path,path.strip("\\"))
                try:
                        lines = read_file_with_encoding_lines(filepath)
                        for orig_l in lines:
                            try:
                                l = orig_l
                                l = self.removeComment(l)
                                l = l.strip()
                                if l == "":
                                    continue
                                if l.startswith("#include "):
                                    include_statements = re.findall(r'^#include\s*[<"][^>"]*[>"]', l)
                                    for include in include_statements:
                                        include = include.replace("/", "\\")
                                        include = re.sub(r"(#include)\s+", "", include)
                                        include = re.sub(r'^["<]|[">]$', '', include)
                                        include = include.strip(";")
                                        include = include.strip()  # 这时应该只剩下引入的头文件相对路径了,不需要补全成绝对路径，因为会和编译和构建系系统的配之有关

                                        if include not in include_dependency_statement:#include引入的可以不是.h后缀的文件
                                            include_dependency_statement.append(orig_l)
                                            dependency_info = {
                                                'statement': include,
                                                'filepath': path
                                            }
                                            if dependency_info not in include_dependency:
                                                include_dependency.append(dependency_info)

                            except Exception as e:
                                print(f"error: {str(e)}")
                                traceback.print_exc()
                                continue
                except Exception as e:
                    print(f"error: {str(e)}")
                    traceback.print_exc()
                    continue
            oss_dependency = {
                'name': oss_name,
                'orig_include_dependency': include_dependency,
                'orig_extern_dependency': extern_dependency,
                'orig_include_dependency_statement': include_dependency_statement,
                'orig_extern_dependency_statement': extern_dependency_statement,
                'include_dependency': include_dependency.copy(),
                'extern_dependency': extern_dependency.copy(),
            }
            dependency_dic.append(oss_dependency)
        return dependency_dic
    def remove_inner_dep(self,target_features, dependency_dic):
        """
        这是先对每个模块内部做一次依赖关系，排除掉一些内部依赖用到的特征和依赖信息
        1.include特征依赖排除：排除掉include内部头文件的语句,去掉的是依赖信息，
        2.extern特征依赖排除:排除掉externvar的变量是内部其他文件定义的externvar信息，排除的也是依赖信息
        3.间接特征依赖排除：对比所有的模块间接依赖特征，只留下各个模块都不同的特征，把相同的特征去掉
        @param target_features:
        @param dependency_dic:
        @return:
        """
        """
        先对上述依赖语句做一次过滤，属于内部的依赖信息就删除，这是为了防止出现误报
        """
        print("消除内部依赖：")
        def split_path(path):
            # 使用split函数按照路径分隔符\切分路径字符串，并去掉空节点
            return [node for node in path.split('\\') if node]

        def is_short_path_contained_in_long_path(short_path, long_path):
            # 分割段路径和长路径
            short_path_nodes = split_path(short_path)
            long_path_nodes = split_path(long_path)

            # 从最后一个节点开始比较
            short_index = len(short_path_nodes) - 1
            long_index = len(long_path_nodes) - 1

            while short_index >= 0 and long_index >= 0:
                if short_path_nodes[short_index] != long_path_nodes[long_index]:
                    # 如果节点不匹配，返回False
                    return False
                short_index -= 1
                long_index -= 1

            # 如果段路径的所有节点都按顺序包含在长路径中，则返回True，否则返回False
            return short_index < 0
        for moudle_dep in dependency_dic:#逐个处理每个模块的依赖信息
            print("消除内部显式依赖")
            oss_name = moudle_dep['name']
            include_dependency_info = moudle_dep['include_dependency']
            extern_dependency_info = moudle_dep['extern_dependency']
            extern_dependency_info_copy=extern_dependency_info.copy()
            include_dependency_info_copy=include_dependency_info.copy()
            for dep in include_dependency_info_copy:#逐个处理该模块的依赖信息
                dep_statement=dep['statement']
                for moudle in target_features:#每个依赖信息都要与所有模块特征做对比
                    moudle_name = moudle['name']
                    if moudle_name==oss_name:#如果被比对的模块是自己本身
                        moudle_feature = moudle['head_file_feature']
                        for future in moudle_feature:#每条依赖信息与模块所有特征做对比
                            if is_short_path_contained_in_long_path(dep_statement,future)is True:
                                include_dependency_info.remove(dep)
                                break

            for dep in extern_dependency_info_copy:  # 逐个处理该模块的依赖信息,注意使用深复制的变量，防止边遍历一边删除元素造成的错误
                if len(dep)==0:
                    continue
                for moudle in target_features:  # 每个依赖信息都要与所有模块特征做对比
                    moudle_name = moudle['name']
                    if moudle_name == oss_name:
                        moudle_feature = moudle['global_feature']
                        if len(moudle_feature)==0:
                            continue
                        for future in moudle_feature:  # 每条依赖信息与模块所有特征做对比
                            if dep['statement_type'] == future['feature_type']:
                                if dep['name']==future['name']:
                                    if dep['type']==future['type']:
                                        if 'signature' in dep :
                                            if  dep['signature'] == future['signature']:
                                               extern_dependency_info.remove(dep)
                                               break
                                        else:
                                            extern_dependency_info.remove(dep)
                                            break
        for moudle_out in target_features:
            print("消除内部间接依赖")
            for moudle_inner in target_features:
                if moudle_out['name']==moudle_inner['name']:
                    continue
                else:
                    out_indirect_feature = moudle_out['indirect_feature'].copy()
                    in_indirect_feature = moudle_inner['indirect_feature'].copy()
                    for out in out_indirect_feature:
                        for inner in in_indirect_feature:
                            if  out['name']== inner['name']:
                                        print(f"有相同标识符名称：{out},{inner}")
                                        if out in  moudle_out['indirect_feature']:
                                            moudle_out['indirect_feature'].remove(out)
                                        if inner in moudle_inner['indirect_feature']:
                                            moudle_inner['indirect_feature'].remove(inner)

        return dependency_dic

    # def main(self,module_dic,target_src_path):
    #     """
    #     target_path:目标源码的路径
    #     module_dic:目标源码划分成的模块
    #     return：target_features:每一个部分的特征
    #     """
    #     target_features = []
    #     dependency_dic = []
    #     target_features=self.headfiels_feature(module_dic,target_features)
    #     dependency_dic=self.scan_include_dependency(module_dic,dependency_dic,target_src_path)
    #     target_features,dependency_dic=self.ctags_other_feature(module_dic,target_features,dependency_dic,target_src_path)
    #     dependency_dic_=self.remove_inner_dep(target_features,dependency_dic)
    #     return target_features,dependency_dic_
    def main(self, module_dic, target_src_path,numprocess=8):
        """
        target_path:目标源码的路径
        module_dic:目标源码划分成的模块
        return：target_features:每一个部分的特征
        """
        target_features = []
        dependency_dic = []
        target_features = self.headfiels_feature(module_dic, target_features)
        dependency_dic = self.scan_include_dependency_main(module_dic,numprocess, dependency_dic,target_src_path)
        target_features, dependency_dic = self.ctags_other_feature_main(module_dic, target_features, dependency_dic,
                                                                   target_src_path,numprocess)
        print(f'查看特征：{target_features}')
        print(f'查看依赖：{dependency_dic}')
        dependency_dic_ = self.remove_inner_dep(target_features, dependency_dic)
        return target_features, dependency_dic_

if __name__ == "__main__":
    module_dic_input=input("moudle_dic_path:")

    module_dic=read_jl(module_dic_input)
    targetpath=input("targetpath:")
    extraction_feature=ExtractionFeature()
    target_features,dependency_dic=extraction_feature.main(module_dic,target_src_path=targetpath)
    print(dependency_dic)
    fres = open("result_dependencystatement.json", 'w')
    for dic in dependency_dic:
        json_str = json.dumps(dic)
        fres.write(json_str + '\n')
    fres = open("result_feature.json", 'w')
    for dic in target_features:
        json_str = json.dumps(dic)
        fres.write(json_str + '\n')
